{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3e172a63",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-09-08T20:30:58.573181Z",
     "iopub.status.busy": "2021-09-08T20:30:58.572015Z",
     "iopub.status.idle": "2021-09-08T20:33:55.432875Z",
     "shell.execute_reply": "2021-09-08T20:33:55.431975Z",
     "shell.execute_reply.started": "2021-09-08T16:08:15.16797Z"
    },
    "papermill": {
     "duration": 176.878848,
     "end_time": "2021-09-08T20:33:55.433073",
     "exception": false,
     "start_time": "2021-09-08T20:30:58.554225",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting package metadata (current_repodata.json): - \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\bdone\r\n",
      "Solving environment: / \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\bdone\r\n",
      "\r\n",
      "\r\n",
      "==> WARNING: A newer version of conda exists. <==\r\n",
      "  current version: 4.9.2\r\n",
      "  latest version: 4.10.3\r\n",
      "\r\n",
      "Please update conda by running\r\n",
      "\r\n",
      "    $ conda update -n base conda\r\n",
      "\r\n",
      "\r\n",
      "\r\n",
      "## Package Plan ##\r\n",
      "\r\n",
      "  environment location: /opt/conda\r\n",
      "\r\n",
      "  added / updated specs:\r\n",
      "    - scipy==1.5.3\r\n",
      "\r\n",
      "\r\n",
      "The following packages will be downloaded:\r\n",
      "\r\n",
      "    package                    |            build\r\n",
      "    ---------------------------|-----------------\r\n",
      "    scipy-1.5.3                |   py37h14a347d_0        19.1 MB  conda-forge\r\n",
      "    ------------------------------------------------------------\r\n",
      "                                           Total:        19.1 MB\r\n",
      "\r\n",
      "The following packages will be DOWNGRADED:\r\n",
      "\r\n",
      "  scipy                                1.6.3-py37h29e03ee_0 --> 1.5.3-py37h14a347d_0\r\n",
      "\r\n",
      "\r\n",
      "\r\n",
      "Downloading and Extracting Packages\r\n",
      "scipy-1.5.3          | 19.1 MB   | ##################################### | 100% \r\n",
      "Preparing transaction: - \b\b\\ \b\bdone\r\n",
      "Verifying transaction: / \b\b- \b\b\\ \b\bdone\r\n",
      "Executing transaction: / \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\bdone\r\n"
     ]
    }
   ],
   "source": [
    "# Fixing a problem with Skopt (see https://github.com/scikit-optimize/scikit-optimize/issues/981)\n",
    "!conda install scipy=='1.5.3' --y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c5ed587b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T20:33:56.083961Z",
     "iopub.status.busy": "2021-09-08T20:33:56.083226Z",
     "iopub.status.idle": "2021-09-08T20:34:04.581708Z",
     "shell.execute_reply": "2021-09-08T20:34:04.580874Z",
     "shell.execute_reply.started": "2021-09-08T16:11:11.899416Z"
    },
    "papermill": {
     "duration": 8.827643,
     "end_time": "2021-09-08T20:34:04.581877",
     "exception": false,
     "start_time": "2021-09-08T20:33:55.754234",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: scikit-learn==0.23.2 in /opt/conda/lib/python3.7/site-packages (0.23.2)\r\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn==0.23.2) (2.1.0)\r\n",
      "Requirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.7/site-packages (from scikit-learn==0.23.2) (1.5.3)\r\n",
      "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn==0.23.2) (1.0.1)\r\n",
      "Requirement already satisfied: numpy>=1.13.3 in /opt/conda/lib/python3.7/site-packages (from scikit-learn==0.23.2) (1.19.5)\r\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!pip install scikit-learn=='0.23.2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9fa8ed64",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T20:34:05.254156Z",
     "iopub.status.busy": "2021-09-08T20:34:05.252904Z",
     "iopub.status.idle": "2021-09-08T20:34:06.189790Z",
     "shell.execute_reply": "2021-09-08T20:34:06.189027Z",
     "shell.execute_reply.started": "2021-09-08T16:11:20.563559Z"
    },
    "papermill": {
     "duration": 1.287065,
     "end_time": "2021-09-08T20:34:06.189981",
     "exception": false,
     "start_time": "2021-09-08T20:34:04.902916",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Importing core libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from time import time\n",
    "import pprint\n",
    "import joblib\n",
    "from functools import partial\n",
    "\n",
    "# Suppressing warnings because of skopt verbosity\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Classifier/Regressor\n",
    "from xgboost import XGBRegressor\n",
    "\n",
    "# Model selection\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Metrics\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import make_scorer\n",
    "\n",
    "# Skopt functions\n",
    "from skopt import BayesSearchCV\n",
    "from skopt.callbacks import DeadlineStopper, DeltaYStopper\n",
    "from skopt.space import Real, Categorical, Integer\n",
    "from skopt.utils import use_named_args # decorator to convert a list of parameters to named arguments\n",
    "from skopt import gp_minimize, forest_minimize\n",
    "from skopt import gbrt_minimize, dummy_minimize\n",
    "\n",
    "# Data processing\n",
    "from sklearn.preprocessing import OrdinalEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "05dc7b72",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T20:34:06.844107Z",
     "iopub.status.busy": "2021-09-08T20:34:06.843329Z",
     "iopub.status.idle": "2021-09-08T20:34:15.945837Z",
     "shell.execute_reply": "2021-09-08T20:34:15.946408Z",
     "shell.execute_reply.started": "2021-09-08T16:11:21.461713Z"
    },
    "papermill": {
     "duration": 9.430822,
     "end_time": "2021-09-08T20:34:15.946675",
     "exception": false,
     "start_time": "2021-09-08T20:34:06.515853",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Loading data \n",
    "X_train = pd.read_csv(\"../input/30-days-of-ml/train.csv\")\n",
    "X_test = pd.read_csv(\"../input/30-days-of-ml/test.csv\")\n",
    "\n",
    "# Preparing data as a tabular matrix\n",
    "y_train = X_train.target\n",
    "X_train = X_train.set_index('id').drop('target', axis='columns')\n",
    "X_test = X_test.set_index('id')\n",
    "\n",
    "# Pointing out categorical features\n",
    "categoricals = [item for item in X_train.columns if 'cat' in item]\n",
    "\n",
    "# Dealing with categorical data using OrdinalEncoder\n",
    "ordinal_encoder = OrdinalEncoder()\n",
    "X_train[categoricals] = ordinal_encoder.fit_transform(X_train[categoricals])\n",
    "X_test[categoricals] = ordinal_encoder.transform(X_test[categoricals])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0ce3c1cf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T20:34:16.629691Z",
     "iopub.status.busy": "2021-09-08T20:34:16.628681Z",
     "iopub.status.idle": "2021-09-08T20:34:16.631863Z",
     "shell.execute_reply": "2021-09-08T20:34:16.632323Z",
     "shell.execute_reply.started": "2021-09-08T16:11:30.817377Z"
    },
    "papermill": {
     "duration": 0.334103,
     "end_time": "2021-09-08T20:34:16.632541",
     "exception": false,
     "start_time": "2021-09-08T20:34:16.298438",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the scoring function\n",
    "scoring = partial(mean_squared_error, squared=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1d35522d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T20:34:17.297162Z",
     "iopub.status.busy": "2021-09-08T20:34:17.296152Z",
     "iopub.status.idle": "2021-09-08T20:34:17.300084Z",
     "shell.execute_reply": "2021-09-08T20:34:17.299280Z",
     "shell.execute_reply.started": "2021-09-08T16:11:30.824028Z"
    },
    "papermill": {
     "duration": 0.338823,
     "end_time": "2021-09-08T20:34:17.300263",
     "exception": false,
     "start_time": "2021-09-08T20:34:16.961440",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the cv strategy\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "da9e90fe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T20:34:17.967317Z",
     "iopub.status.busy": "2021-09-08T20:34:17.966624Z",
     "iopub.status.idle": "2021-09-08T20:34:17.970086Z",
     "shell.execute_reply": "2021-09-08T20:34:17.969475Z",
     "shell.execute_reply.started": "2021-09-08T16:11:30.83955Z"
    },
    "papermill": {
     "duration": 0.347506,
     "end_time": "2021-09-08T20:34:17.970273",
     "exception": false,
     "start_time": "2021-09-08T20:34:17.622767",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the search space\n",
    "space = [Real(0.01, 1.0, 'uniform', name='learning_rate'),\n",
    "         Integer(1, 8, name='max_depth'),\n",
    "         Real(0.1, 1.0, 'uniform', name='subsample'),\n",
    "         Real(0.1, 1.0, 'uniform', name='colsample_bytree'),  # subsample ratio of columns by tree\n",
    "         Real(0, 100., 'uniform', name='reg_lambda'),      # L2 regularization\n",
    "         Real(0, 100., 'uniform', name='reg_alpha'),       # L1 regularization\n",
    "         Real(1, 30, 'uniform', name='min_child_weight'),     # minimum sum of instance weight (hessian)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8a594e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = XGBRegressor(n_estimators=10_000, \n",
    "                     booster='gbtree', random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1c5b34db",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T20:34:18.639076Z",
     "iopub.status.busy": "2021-09-08T20:34:18.638176Z",
     "iopub.status.idle": "2021-09-08T20:34:18.649274Z",
     "shell.execute_reply": "2021-09-08T20:34:18.649805Z",
     "shell.execute_reply.started": "2021-09-08T16:11:30.857578Z"
    },
    "papermill": {
     "duration": 0.345123,
     "end_time": "2021-09-08T20:34:18.650019",
     "exception": false,
     "start_time": "2021-09-08T20:34:18.304896",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# The objective function to be minimized\n",
    "def make_objective(model, X, y, space, cv, scoring, validation=0.2):\n",
    "    # This decorator converts your objective function with named arguments into one that\n",
    "    # accepts a list as argument, while doing the conversion automatically.\n",
    "    @use_named_args(space) \n",
    "    def objective(**params):\n",
    "        model.set_params(**params)\n",
    "        print(\"\\nTesting: \", params)\n",
    "        validation_scores = list()\n",
    "        for k, (train_index, test_index) in enumerate(kf.split(X, y)):\n",
    "            val_index = list()\n",
    "            train_examples = len(train_index)\n",
    "            train_examples = int(train_examples * (1 - validation))\n",
    "            train_index, val_index = train_index[:train_examples], train_index[train_examples:]\n",
    "            \n",
    "            start_time = time()\n",
    "            model.fit(X.iloc[train_index,:], y[train_index],\n",
    "                      early_stopping_rounds=50,\n",
    "                      eval_set=[(X.iloc[val_index,:], y[val_index])], \n",
    "                      verbose=0\n",
    "                    )\n",
    "            end_time = time()\n",
    "            \n",
    "            rounds = model.best_iteration\n",
    "            \n",
    "            test_preds = model.predict(X.iloc[test_index,:])\n",
    "            test_score = scoring(y[test_index], test_preds)\n",
    "            print(f\"CV Fold {k+1} rmse:{test_score:0.5f} - {rounds} rounds - it took {end_time-start_time:0.0f} secs\")\n",
    "            validation_scores.append(test_score)\n",
    "            \n",
    "            if len(history[k]) >= 10:\n",
    "                threshold = np.percentile(history[k], q=25)\n",
    "                if test_score > threshold:\n",
    "                    print(f\"Early stopping for under-performing fold: threshold is {threshold:0.5f}\")\n",
    "                    return np.mean(validation_scores)\n",
    "                \n",
    "            history[k].append(test_score)\n",
    "        return np.mean(validation_scores)\n",
    "\n",
    "    return objective"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b5a48022",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T20:34:20.002041Z",
     "iopub.status.busy": "2021-09-08T20:34:20.001261Z",
     "iopub.status.idle": "2021-09-08T20:34:20.005203Z",
     "shell.execute_reply": "2021-09-08T20:34:20.004559Z",
     "shell.execute_reply.started": "2021-09-08T16:11:30.889487Z"
    },
    "papermill": {
     "duration": 0.330822,
     "end_time": "2021-09-08T20:34:20.005361",
     "exception": false,
     "start_time": "2021-09-08T20:34:19.674539",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "objective = make_objective(model,\n",
    "                           X_train, y_train,\n",
    "                           space=space,\n",
    "                           cv=kf,\n",
    "                           scoring=scoring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3827911",
   "metadata": {},
   "outputs": [],
   "source": [
    "def onstep(res):\n",
    "    global counter\n",
    "    x0 = res.x_iters   # List of input points\n",
    "    y0 = res.func_vals # Evaluation of input points\n",
    "    print('Last eval: ', x0[-1], \n",
    "          ' - Score ', y0[-1])\n",
    "    print('Current iter: ', counter, \n",
    "          ' - Best Score ', res.fun, \n",
    "          ' - Best Args: ', res.x)\n",
    "    joblib.dump((x0, y0), 'checkpoint.pkl') # Saving a checkpoint to disk\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da332fd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "history = {i:list() for i in range(5)}\n",
    "used_time = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6a496030",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T20:34:20.648652Z",
     "iopub.status.busy": "2021-09-08T20:34:20.647942Z",
     "iopub.status.idle": "2021-09-08T22:10:17.477821Z",
     "shell.execute_reply": "2021-09-08T22:10:17.478490Z"
    },
    "papermill": {
     "duration": 5757.152824,
     "end_time": "2021-09-08T22:10:17.478899",
     "exception": false,
     "start_time": "2021-09-08T20:34:20.326075",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing:  {'learning_rate': 0.5969161720427683, 'max_depth': 1, 'subsample': 0.6424870384644795, 'colsample_bytree': 0.5903948646972073, 'reg_lambda': 42.36547993389048, 'reg_alpha': 64.58941130666562, 'min_child_weight': 13.690029126618084}\n",
      "CV Fold 1 rmse:0.72157 - 2190 rounds - it took 119 secs\n",
      "CV Fold 2 rmse:0.71872 - 2307 rounds - it took 124 secs\n",
      "CV Fold 3 rmse:0.72394 - 2166 rounds - it took 116 secs\n",
      "CV Fold 4 rmse:0.71826 - 1681 rounds - it took 91 secs\n",
      "CV Fold 5 rmse:0.71747 - 2670 rounds - it took 143 secs\n",
      "Last eval:  [0.5969161720427683, 1, 0.6424870384644795, 0.5903948646972073, 42.36547993389048, 64.58941130666562, 13.690029126618084]  - Score  0.7199923709490351\n",
      "Current iter:  0  - Best Score  0.7199923709490351  - Best Args:  [0.5969161720427683, 1, 0.6424870384644795, 0.5903948646972073, 42.36547993389048, 64.58941130666562, 13.690029126618084]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8928552707742591, 'max_depth': 1, 'subsample': 0.3453906651221019, 'colsample_bytree': 0.529898605589215, 'reg_lambda': 81.21687287754933, 'reg_alpha': 47.99771723750574, 'min_child_weight': 12.390759086924064}\n",
      "CV Fold 1 rmse:0.72394 - 702 rounds - it took 31 secs\n",
      "CV Fold 2 rmse:0.71961 - 1067 rounds - it took 45 secs\n",
      "CV Fold 3 rmse:0.72642 - 608 rounds - it took 27 secs\n",
      "CV Fold 4 rmse:0.71816 - 956 rounds - it took 40 secs\n",
      "CV Fold 5 rmse:0.71903 - 871 rounds - it took 37 secs\n",
      "Last eval:  [0.8928552707742591, 1, 0.3453906651221019, 0.529898605589215, 81.21687287754933, 47.99771723750574, 12.390759086924064]  - Score  0.7214318434822131\n",
      "Current iter:  1  - Best Score  0.7199923709490351  - Best Args:  [0.5969161720427683, 1, 0.6424870384644795, 0.5903948646972073, 42.36547993389048, 64.58941130666562, 13.690029126618084]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8377179759020038, 'max_depth': 1, 'subsample': 0.17841636973138664, 'colsample_bytree': 0.11819655769629316, 'reg_lambda': 83.26198455479381, 'reg_alpha': 77.81567509498505, 'min_child_weight': 26.230352299157758}\n",
      "CV Fold 1 rmse:0.72561 - 1242 rounds - it took 33 secs\n",
      "CV Fold 2 rmse:0.72388 - 886 rounds - it took 24 secs\n",
      "CV Fold 3 rmse:0.72824 - 1300 rounds - it took 34 secs\n",
      "CV Fold 4 rmse:0.72481 - 482 rounds - it took 13 secs\n",
      "CV Fold 5 rmse:0.72341 - 854 rounds - it took 23 secs\n",
      "Last eval:  [0.8377179759020038, 1, 0.17841636973138664, 0.11819655769629316, 83.26198455479381, 77.81567509498505, 26.230352299157758]  - Score  0.7251892677586431\n",
      "Current iter:  2  - Best Score  0.7199923709490351  - Best Args:  [0.5969161720427683, 1, 0.6424870384644795, 0.5903948646972073, 42.36547993389048, 64.58941130666562, 13.690029126618084]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9788321588104365, 'max_depth': 1, 'subsample': 0.8208196767816799, 'colsample_bytree': 0.5684297315960845, 'reg_lambda': 67.88795301189604, 'reg_alpha': 72.06326547259168, 'min_child_weight': 17.87857397017811}\n",
      "CV Fold 1 rmse:0.72082 - 2112 rounds - it took 115 secs\n",
      "CV Fold 2 rmse:0.71871 - 1669 rounds - it took 91 secs\n",
      "CV Fold 3 rmse:0.72422 - 1382 rounds - it took 77 secs\n",
      "CV Fold 4 rmse:0.71672 - 2146 rounds - it took 116 secs\n",
      "CV Fold 5 rmse:0.71768 - 1834 rounds - it took 100 secs\n",
      "Last eval:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]  - Score  0.7196308890047669\n",
      "Current iter:  3  - Best Score  0.7196308890047669  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.5419994971545207, 'max_depth': 1, 'subsample': 0.5696634895750646, 'colsample_bytree': 0.4731957459914713, 'reg_lambda': 26.4555612104627, 'reg_alpha': 77.42336894342168, 'min_child_weight': 14.22835963427991}\n",
      "CV Fold 1 rmse:0.72175 - 2440 rounds - it took 114 secs\n",
      "CV Fold 2 rmse:0.72017 - 1774 rounds - it took 83 secs\n",
      "CV Fold 3 rmse:0.72414 - 2459 rounds - it took 115 secs\n",
      "CV Fold 4 rmse:0.71869 - 1758 rounds - it took 83 secs\n",
      "CV Fold 5 rmse:0.71784 - 2975 rounds - it took 138 secs\n",
      "Last eval:  [0.5419994971545207, 1, 0.5696634895750646, 0.4731957459914713, 26.4555612104627, 77.42336894342168, 14.22835963427991]  - Score  0.720516924298628\n",
      "Current iter:  4  - Best Score  0.7196308890047669  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.5727496093799621, 'max_depth': 1, 'subsample': 0.3917269070138927, 'colsample_bytree': 0.2347073804653149, 'reg_lambda': 22.232138825158767, 'reg_alpha': 38.6488981125862, 'min_child_weight': 27.17535579035274}\n",
      "CV Fold 1 rmse:0.72256 - 1588 rounds - it took 53 secs\n",
      "CV Fold 2 rmse:0.72031 - 1370 rounds - it took 47 secs\n",
      "CV Fold 3 rmse:0.72500 - 1691 rounds - it took 57 secs\n",
      "CV Fold 4 rmse:0.71947 - 1209 rounds - it took 42 secs\n",
      "CV Fold 5 rmse:0.71838 - 1990 rounds - it took 67 secs\n",
      "Last eval:  [0.5727496093799621, 1, 0.3917269070138927, 0.2347073804653149, 22.232138825158767, 38.6488981125862, 27.17535579035274]  - Score  0.7211427697045606\n",
      "Current iter:  5  - Best Score  0.7196308890047669  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.45545049001211535, 'max_depth': 8, 'subsample': 0.4933287584194074, 'colsample_bytree': 0.7278680763345384, 'reg_lambda': 6.022547162926984, 'reg_alpha': 66.67667154456677, 'min_child_weight': 20.448498218926627}\n",
      "CV Fold 1 rmse:0.72767 - 56 rounds - it took 39 secs\n",
      "CV Fold 2 rmse:0.72484 - 61 rounds - it took 40 secs\n",
      "CV Fold 3 rmse:0.72992 - 57 rounds - it took 39 secs\n",
      "CV Fold 4 rmse:0.72442 - 66 rounds - it took 42 secs\n",
      "CV Fold 5 rmse:0.72436 - 43 rounds - it took 34 secs\n",
      "Last eval:  [0.45545049001211535, 8, 0.4933287584194074, 0.7278680763345384, 6.022547162926984, 66.67667154456677, 20.448498218926627]  - Score  0.7262420082695538\n",
      "Current iter:  6  - Best Score  0.7196308890047669  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.21827873546310253, 'max_depth': 4, 'subsample': 0.7756175270966107, 'colsample_bytree': 0.6470476018439211, 'reg_lambda': 32.504722900835255, 'reg_alpha': 3.842542647273473, 'min_child_weight': 19.393947680762718}\n",
      "CV Fold 1 rmse:0.72225 - 362 rounds - it took 76 secs\n",
      "CV Fold 2 rmse:0.71978 - 357 rounds - it took 75 secs\n",
      "CV Fold 3 rmse:0.72327 - 392 rounds - it took 82 secs\n",
      "CV Fold 4 rmse:0.71770 - 303 rounds - it took 65 secs\n",
      "CV Fold 5 rmse:0.71827 - 360 rounds - it took 75 secs\n",
      "Last eval:  [0.21827873546310253, 4, 0.7756175270966107, 0.6470476018439211, 32.504722900835255, 3.842542647273473, 19.393947680762718]  - Score  0.7202548668750725\n",
      "Current iter:  7  - Best Score  0.7196308890047669  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9593597759382753, 'max_depth': 8, 'subsample': 0.28798908048535127, 'colsample_bytree': 0.24517856609649666, 'reg_lambda': 65.31083254653986, 'reg_alpha': 25.329160253978216, 'min_child_weight': 14.523012412832884}\n",
      "CV Fold 1 rmse:0.73717 - 9 rounds - it took 9 secs\n",
      "CV Fold 2 rmse:0.73289 - 13 rounds - it took 9 secs\n",
      "CV Fold 3 rmse:0.73800 - 13 rounds - it took 10 secs\n",
      "CV Fold 4 rmse:0.73195 - 9 rounds - it took 9 secs\n",
      "CV Fold 5 rmse:0.73261 - 9 rounds - it took 9 secs\n",
      "Last eval:  [0.9593597759382753, 8, 0.28798908048535127, 0.24517856609649666, 65.31083254653986, 25.329160253978216, 14.523012412832884]  - Score  0.7345241075961837\n",
      "Current iter:  8  - Best Score  0.7196308890047669  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.25198133608158674, 'max_depth': 8, 'subsample': 0.4042068533550026, 'colsample_bytree': 0.7072770900331187, 'reg_lambda': 31.720174206929613, 'reg_alpha': 77.83454820259091, 'min_child_weight': 28.537560550071525}\n",
      "CV Fold 1 rmse:0.72570 - 164 rounds - it took 64 secs\n",
      "CV Fold 2 rmse:0.72346 - 119 rounds - it took 51 secs\n",
      "CV Fold 3 rmse:0.72775 - 208 rounds - it took 77 secs\n",
      "CV Fold 4 rmse:0.72150 - 172 rounds - it took 66 secs\n",
      "CV Fold 5 rmse:0.72235 - 192 rounds - it took 72 secs\n",
      "Last eval:  [0.25198133608158674, 8, 0.4042068533550026, 0.7072770900331187, 31.720174206929613, 77.83454820259091, 28.537560550071525]  - Score  0.7241524444148069\n",
      "Current iter:  9  - Best Score  0.7196308890047669  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.6659015982805438, 'max_depth': 7, 'subsample': 0.18739114821375516, 'colsample_bytree': 0.8541504167489236, 'reg_lambda': 9.609840789396308, 'reg_alpha': 97.6459465013396, 'min_child_weight': 14.590884847783347}\n",
      "CV Fold 1 rmse:0.73270 - 54 rounds - it took 22 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72187\n",
      "Last eval:  [0.6659015982805438, 7, 0.18739114821375516, 0.8541504167489236, 9.609840789396308, 97.6459465013396, 14.590884847783347]  - Score  0.7327047959456201\n",
      "Current iter:  10  - Best Score  0.7196308890047669  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9769934773084339, 'max_depth': 8, 'subsample': 0.5060432930688353, 'colsample_bytree': 0.11798889886788287, 'reg_lambda': 44.171092124884545, 'reg_alpha': 97.95867288127286, 'min_child_weight': 11.423889455110325}\n",
      "CV Fold 1 rmse:0.72361 - 234 rounds - it took 27 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72187\n",
      "Last eval:  [0.9769934773084339, 8, 0.5060432930688353, 0.11798889886788287, 44.171092124884545, 97.95867288127286, 11.423889455110325]  - Score  0.7236097230689771\n",
      "Current iter:  11  - Best Score  0.7196308890047669  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.48608459552780126, 'max_depth': 2, 'subsample': 0.3861848614545784, 'colsample_bytree': 0.472836695063203, 'reg_lambda': 6.414749634878437, 'reg_alpha': 69.24721193700199, 'min_child_weight': 17.431442171990682}\n",
      "CV Fold 1 rmse:0.72231 - 689 rounds - it took 49 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72187\n",
      "Last eval:  [0.48608459552780126, 2, 0.3861848614545784, 0.472836695063203, 6.414749634878437, 69.24721193700199, 17.431442171990682]  - Score  0.722308040366438\n",
      "Current iter:  12  - Best Score  0.7196308890047669  - Best Args:  [0.9788321588104365, 1, 0.8208196767816799, 0.5684297315960845, 67.88795301189604, 72.06326547259168, 17.87857397017811]\n",
      "\n",
      "Testing:  {'learning_rate': 0.272735596030051, 'max_depth': 4, 'subsample': 0.9250506586175148, 'colsample_bytree': 0.9290418492134799, 'reg_lambda': 8.31124926306024, 'reg_alpha': 27.77185612810325, 'min_child_weight': 1.2713444408394459}\n",
      "CV Fold 1 rmse:0.72123 - 298 rounds - it took 94 secs\n",
      "CV Fold 2 rmse:0.71900 - 307 rounds - it took 96 secs\n",
      "CV Fold 3 rmse:0.72295 - 326 rounds - it took 103 secs\n",
      "CV Fold 4 rmse:0.71709 - 386 rounds - it took 119 secs\n",
      "CV Fold 5 rmse:0.71784 - 303 rounds - it took 95 secs\n",
      "Last eval:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]  - Score  0.7196204717648156\n",
      "Current iter:  13  - Best Score  0.7196204717648156  - Best Args:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8439186588710538, 'max_depth': 1, 'subsample': 0.7446944837067091, 'colsample_bytree': 0.360465483652481, 'reg_lambda': 18.319136200711686, 'reg_alpha': 58.651293481008324, 'min_child_weight': 1.5831188394373132}\n",
      "CV Fold 1 rmse:0.72256 - 1265 rounds - it took 54 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72166\n",
      "Last eval:  [0.8439186588710538, 1, 0.7446944837067091, 0.360465483652481, 18.319136200711686, 58.651293481008324, 1.5831188394373132]  - Score  0.7225607537772744\n",
      "Current iter:  14  - Best Score  0.7196204717648156  - Best Args:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8306506289251896, 'max_depth': 7, 'subsample': 0.23179758645620105, 'colsample_bytree': 0.6126565653309761, 'reg_lambda': 70.37372792899164, 'reg_alpha': 28.84764370485287, 'min_child_weight': 13.565353797479279}\n",
      "CV Fold 1 rmse:0.73682 - 10 rounds - it took 14 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72166\n",
      "Last eval:  [0.8306506289251896, 7, 0.23179758645620105, 0.6126565653309761, 70.37372792899164, 28.84764370485287, 13.565353797479279]  - Score  0.7368213889610651\n",
      "Current iter:  15  - Best Score  0.7196204717648156  - Best Args:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]\n",
      "\n",
      "Testing:  {'learning_rate': 0.7585456269263906, 'max_depth': 8, 'subsample': 0.6328377381446552, 'colsample_bytree': 0.6150267152117861, 'reg_lambda': 22.308163264061836, 'reg_alpha': 95.27490115169851, 'min_child_weight': 13.966635979911194}\n",
      "CV Fold 1 rmse:0.72818 - 39 rounds - it took 30 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72166\n",
      "Last eval:  [0.7585456269263906, 8, 0.6328377381446552, 0.6150267152117861, 22.308163264061836, 95.27490115169851, 13.966635979911194]  - Score  0.7281835109168071\n",
      "Current iter:  16  - Best Score  0.7196204717648156  - Best Args:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8479445857464166, 'max_depth': 6, 'subsample': 0.204581718651155, 'colsample_bytree': 0.7903213332028541, 'reg_lambda': 41.18201389648448, 'reg_alpha': 67.5439081200236, 'min_child_weight': 8.244091993994365}\n",
      "CV Fold 1 rmse:0.73618 - 12 rounds - it took 12 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72166\n",
      "Last eval:  [0.8479445857464166, 6, 0.204581718651155, 0.7903213332028541, 41.18201389648448, 67.5439081200236, 8.244091993994365]  - Score  0.7361799382509663\n",
      "Current iter:  17  - Best Score  0.7196204717648156  - Best Args:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]\n",
      "\n",
      "Testing:  {'learning_rate': 0.3200861486264915, 'max_depth': 7, 'subsample': 0.7232784310699893, 'colsample_bytree': 0.7527288518376766, 'reg_lambda': 50.132438192670236, 'reg_alpha': 95.6083634723224, 'min_child_weight': 19.675715777659487}\n",
      "CV Fold 1 rmse:0.72418 - 155 rounds - it took 80 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72166\n",
      "Last eval:  [0.3200861486264915, 7, 0.7232784310699893, 0.7527288518376766, 50.132438192670236, 95.6083634723224, 19.675715777659487]  - Score  0.7241820391164526\n",
      "Current iter:  18  - Best Score  0.7196204717648156  - Best Args:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]\n",
      "\n",
      "Testing:  {'learning_rate': 0.42961649807259794, 'max_depth': 4, 'subsample': 0.5273807638680479, 'colsample_bytree': 0.5231189703301887, 'reg_lambda': 71.60745312286433, 'reg_alpha': 28.799100436328054, 'min_child_weight': 12.12040452601349}\n",
      "CV Fold 1 rmse:0.72345 - 146 rounds - it took 28 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72166\n",
      "Last eval:  [0.42961649807259794, 4, 0.5273807638680479, 0.5231189703301887, 71.60745312286433, 28.799100436328054, 12.12040452601349]  - Score  0.7234470452915568\n",
      "Current iter:  19  - Best Score  0.7196204717648156  - Best Args:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]\n",
      "\n",
      "Testing:  {'learning_rate': 0.7516781387811975, 'max_depth': 1, 'subsample': 0.22192665780020523, 'colsample_bytree': 0.3684540933604278, 'reg_lambda': 56.996491070126496, 'reg_alpha': 59.08727612481733, 'min_child_weight': 17.655432216637788}\n",
      "CV Fold 1 rmse:0.72369 - 966 rounds - it took 32 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72166\n",
      "Last eval:  [0.7516781387811975, 1, 0.22192665780020523, 0.3684540933604278, 56.996491070126496, 59.08727612481733, 17.655432216637788]  - Score  0.7236863660566907\n",
      "Current iter:  20  - Best Score  0.7196204717648156  - Best Args:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]\n",
      "\n",
      "Testing:  {'learning_rate': 0.6566688116585624, 'max_depth': 6, 'subsample': 0.38889751723614785, 'colsample_bytree': 0.2991364166654045, 'reg_lambda': 14.126390492589671, 'reg_alpha': 9.72599270631739, 'min_child_weight': 29.537224999560248}\n",
      "CV Fold 1 rmse:0.73240 - 21 rounds - it took 11 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72166\n",
      "Last eval:  [0.6566688116585624, 6, 0.38889751723614785, 0.2991364166654045, 14.126390492589671, 9.72599270631739, 29.537224999560248]  - Score  0.7324045401774013\n",
      "Current iter:  21  - Best Score  0.7196204717648156  - Best Args:  [0.272735596030051, 4, 0.9250506586175148, 0.9290418492134799, 8.31124926306024, 27.77185612810325, 1.2713444408394459]\n",
      "\n",
      "Testing:  {'learning_rate': 0.26773751831153064, 'max_depth': 6, 'subsample': 0.7334997251863298, 'colsample_bytree': 0.19020419858107102, 'reg_lambda': 91.94826137446736, 'reg_alpha': 71.42412995491115, 'min_child_weight': 29.966563190468133}\n",
      "CV Fold 1 rmse:0.72103 - 310 rounds - it took 45 secs\n",
      "CV Fold 2 rmse:0.71806 - 272 rounds - it took 40 secs\n",
      "CV Fold 3 rmse:0.72298 - 271 rounds - it took 40 secs\n",
      "CV Fold 4 rmse:0.71646 - 341 rounds - it took 49 secs\n",
      "CV Fold 5 rmse:0.71759 - 415 rounds - it took 59 secs\n",
      "Last eval:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]  - Score  0.7192252469331295\n",
      "Current iter:  22  - Best Score  0.7192252469331295  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.15795382161141383, 'max_depth': 8, 'subsample': 0.13380344544532627, 'colsample_bytree': 0.5574839037210019, 'reg_lambda': 16.684751304849968, 'reg_alpha': 77.90510196090358, 'min_child_weight': 26.08306557247229}\n",
      "CV Fold 1 rmse:0.72644 - 431 rounds - it took 77 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72149\n",
      "Last eval:  [0.15795382161141383, 8, 0.13380344544532627, 0.5574839037210019, 16.684751304849968, 77.90510196090358, 26.08306557247229]  - Score  0.7264363358146884\n",
      "Current iter:  23  - Best Score  0.7192252469331295  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.41728275651790003, 'max_depth': 4, 'subsample': 0.4664649675033997, 'colsample_bytree': 0.16225029590962425, 'reg_lambda': 69.74287731445638, 'reg_alpha': 45.354268267806894, 'min_child_weight': 21.939612384640093}\n",
      "CV Fold 1 rmse:0.72137 - 365 rounds - it took 31 secs\n",
      "CV Fold 2 rmse:0.71914 - 266 rounds - it took 24 secs\n",
      "Early stopping for under-performing fold: threshold is 0.71893\n",
      "Last eval:  [0.41728275651790003, 4, 0.4664649675033997, 0.16225029590962425, 69.74287731445638, 45.354268267806894, 21.939612384640093]  - Score  0.7202563792238579\n",
      "Current iter:  24  - Best Score  0.7192252469331295  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.867718502669343, 'max_depth': 8, 'subsample': 0.11496517636048054, 'colsample_bytree': 0.30766810186611615, 'reg_lambda': 76.49116989968573, 'reg_alpha': 94.41235194354547, 'min_child_weight': 22.749978214332913}\n",
      "CV Fold 1 rmse:0.73375 - 41 rounds - it took 10 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72137\n",
      "Last eval:  [0.867718502669343, 8, 0.11496517636048054, 0.30766810186611615, 76.49116989968573, 94.41235194354547, 22.749978214332913]  - Score  0.7337457905608021\n",
      "Current iter:  25  - Best Score  0.7192252469331295  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.34600978125502346, 'max_depth': 1, 'subsample': 0.14890418950532827, 'colsample_bytree': 0.2799968724067601, 'reg_lambda': 1.8521794460613974, 'reg_alpha': 79.36977033574207, 'min_child_weight': 7.4938159537510245}\n",
      "CV Fold 1 rmse:0.72389 - 2994 rounds - it took 85 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72137\n",
      "Last eval:  [0.34600978125502346, 1, 0.14890418950532827, 0.2799968724067601, 1.8521794460613974, 79.36977033574207, 7.4938159537510245]  - Score  0.7238927236564209\n",
      "Current iter:  26  - Best Score  0.7192252469331295  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.3518981638899337, 'max_depth': 1, 'subsample': 0.6474275792686117, 'colsample_bytree': 0.6369898656655478, 'reg_lambda': 78.36442453018886, 'reg_alpha': 50.00262977293015, 'min_child_weight': 2.4607316459687705}\n",
      "CV Fold 1 rmse:0.72343 - 2053 rounds - it took 114 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72137\n",
      "Last eval:  [0.3518981638899337, 1, 0.6474275792686117, 0.6369898656655478, 78.36442453018886, 50.00262977293015, 2.4607316459687705]  - Score  0.7234294949489859\n",
      "Current iter:  27  - Best Score  0.7192252469331295  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.7021070939713261, 'max_depth': 1, 'subsample': 0.9407925981323145, 'colsample_bytree': 0.6525693603693065, 'reg_lambda': 53.56328030249584, 'reg_alpha': 58.990997635457106, 'min_child_weight': 22.173538855986322}\n",
      "CV Fold 1 rmse:0.72071 - 3841 rounds - it took 231 secs\n",
      "CV Fold 2 rmse:0.71798 - 3949 rounds - it took 240 secs\n",
      "CV Fold 3 rmse:0.72286 - 4167 rounds - it took 253 secs\n",
      "CV Fold 4 rmse:0.71709 - 3147 rounds - it took 192 secs\n",
      "CV Fold 5 rmse:0.71748 - 3683 rounds - it took 221 secs\n",
      "Last eval:  [0.7021070939713261, 1, 0.9407925981323145, 0.6525693603693065, 53.56328030249584, 58.990997635457106, 22.173538855986322]  - Score  0.7192252966307128\n",
      "Current iter:  28  - Best Score  0.7192252469331295  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.3188255455248059, 'max_depth': 6, 'subsample': 0.29115144900127776, 'colsample_bytree': 0.3001989019481369, 'reg_lambda': 21.874937373677188, 'reg_alpha': 56.95735345747381, 'min_child_weight': 14.111162015969347}\n",
      "CV Fold 1 rmse:0.72515 - 147 rounds - it took 25 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72127\n",
      "Last eval:  [0.3188255455248059, 6, 0.29115144900127776, 0.3001989019481369, 21.874937373677188, 56.95735345747381, 14.111162015969347]  - Score  0.725149009914586\n",
      "Current iter:  29  - Best Score  0.7192252469331295  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n"
     ]
    }
   ],
   "source": [
    "gp_round = dummy_minimize(func=objective,\n",
    "                          dimensions=space,\n",
    "                          n_calls=30,\n",
    "                          callback=[onstep],\n",
    "                          random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bc78feab",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T22:10:18.268244Z",
     "iopub.status.busy": "2021-09-08T22:10:18.267625Z",
     "iopub.status.idle": "2021-09-08T22:10:18.277837Z",
     "shell.execute_reply": "2021-09-08T22:10:18.277144Z"
    },
    "papermill": {
     "duration": 0.364665,
     "end_time": "2021-09-08T22:10:18.277979",
     "exception": false,
     "start_time": "2021-09-08T22:10:17.913314",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n"
     ]
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "print(len(x0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "67f9e8b3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-08T22:10:19.082018Z",
     "iopub.status.busy": "2021-09-08T22:10:19.078033Z",
     "iopub.status.idle": "2021-09-09T04:44:45.277800Z",
     "shell.execute_reply": "2021-09-09T04:44:45.278459Z"
    },
    "papermill": {
     "duration": 23666.636538,
     "end_time": "2021-09-09T04:44:45.278795",
     "exception": false,
     "start_time": "2021-09-08T22:10:18.642257",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Last eval:  [0.3188255455248059, 6, 0.29115144900127776, 0.3001989019481369, 21.874937373677188, 56.95735345747381, 14.111162015969347]  - Score  0.725149009914586\n",
      "Current iter:  30  - Best Score  0.7192252469331295  - Best Args:  [0.26773751831153064, 6, 0.7334997251863298, 0.19020419858107102, 91.94826137446736, 71.42412995491115, 29.966563190468133]\n",
      "\n",
      "Testing:  {'learning_rate': 0.04750495343906687, 'max_depth': 4, 'subsample': 0.868163246780575, 'colsample_bytree': 0.11711450684675054, 'reg_lambda': 80.20861835126621, 'reg_alpha': 84.49765104431548, 'min_child_weight': 12.063897523196292}\n",
      "CV Fold 1 rmse:0.71945 - 5956 rounds - it took 454 secs\n",
      "CV Fold 2 rmse:0.71676 - 5201 rounds - it took 394 secs\n",
      "CV Fold 3 rmse:0.72145 - 4923 rounds - it took 375 secs\n",
      "CV Fold 4 rmse:0.71493 - 5251 rounds - it took 398 secs\n",
      "CV Fold 5 rmse:0.71628 - 4407 rounds - it took 334 secs\n",
      "Last eval:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]  - Score  0.7177736524943502\n",
      "Current iter:  31  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.08734594975076383, 'max_depth': 8, 'subsample': 0.9403592155057442, 'colsample_bytree': 0.13946518638031843, 'reg_lambda': 3.025821134756269, 'reg_alpha': 88.85488190528768, 'min_child_weight': 2.790093831745776}\n",
      "CV Fold 1 rmse:0.72000 - 1169 rounds - it took 188 secs\n",
      "CV Fold 2 rmse:0.71702 - 1150 rounds - it took 185 secs\n",
      "CV Fold 3 rmse:0.72200 - 893 rounds - it took 146 secs\n",
      "CV Fold 4 rmse:0.71547 - 1078 rounds - it took 172 secs\n",
      "CV Fold 5 rmse:0.71655 - 1070 rounds - it took 175 secs\n",
      "Last eval:  [0.08734594975076383, 8, 0.9403592155057442, 0.13946518638031843, 3.025821134756269, 88.85488190528768, 2.790093831745776]  - Score  0.7182089438587392\n",
      "Current iter:  32  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 6, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 48.879563885538495, 'reg_alpha': 0.0, 'min_child_weight': 1.0}\n",
      "CV Fold 1 rmse:0.71999 - 5101 rounds - it took 540 secs\n",
      "CV Fold 2 rmse:0.71774 - 5053 rounds - it took 530 secs\n",
      "CV Fold 3 rmse:0.72223 - 5055 rounds - it took 539 secs\n",
      "CV Fold 4 rmse:0.71617 - 5183 rounds - it took 556 secs\n",
      "CV Fold 5 rmse:0.71651 - 5323 rounds - it took 566 secs\n",
      "Last eval:  [0.01, 6, 1.0, 0.1, 48.879563885538495, 0.0, 1.0]  - Score  0.7185292808041922\n",
      "Current iter:  33  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.1020501069386032, 'max_depth': 8, 'subsample': 0.9970772496666679, 'colsample_bytree': 0.8368486183052445, 'reg_lambda': 0.9636600269540943, 'reg_alpha': 2.7289102924200956, 'min_child_weight': 11.80129540638929}\n",
      "CV Fold 1 rmse:0.72440 - 182 rounds - it took 118 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72082\n",
      "Last eval:  [0.1020501069386032, 8, 0.9970772496666679, 0.8368486183052445, 0.9636600269540943, 2.7289102924200956, 11.80129540638929]  - Score  0.7244045966072448\n",
      "Current iter:  34  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.5466482872837872, 'max_depth': 6, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 43.18231405357227, 'reg_alpha': 100.0, 'min_child_weight': 1.0}\n",
      "CV Fold 1 rmse:0.72018 - 306 rounds - it took 35 secs\n",
      "CV Fold 2 rmse:0.71778 - 270 rounds - it took 32 secs\n",
      "CV Fold 3 rmse:0.72228 - 414 rounds - it took 38 secs\n",
      "CV Fold 4 rmse:0.71608 - 325 rounds - it took 35 secs\n",
      "CV Fold 5 rmse:0.71699 - 265 rounds - it took 31 secs\n",
      "Last eval:  [0.5466482872837872, 6, 1.0, 0.1, 43.18231405357227, 100.0, 1.0]  - Score  0.718659889069599\n",
      "Current iter:  35  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 6, 'subsample': 0.6088702494257244, 'colsample_bytree': 0.1, 'reg_lambda': 64.05720147462661, 'reg_alpha': 100.0, 'min_child_weight': 1.0}\n",
      "CV Fold 1 rmse:0.72100 - 9993 rounds - it took 916 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72074\n",
      "Last eval:  [0.01, 6, 0.6088702494257244, 0.1, 64.05720147462661, 100.0, 1.0]  - Score  0.7210008794486271\n",
      "Current iter:  36  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 1, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 78.97660412036738, 'reg_alpha': 0.0, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.73246 - 9994 rounds - it took 284 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72074\n",
      "Last eval:  [0.01, 1, 1.0, 0.1, 78.97660412036738, 0.0, 30.0]  - Score  0.7324618296305416\n",
      "Current iter:  37  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9802376329369777, 'max_depth': 3, 'subsample': 0.9499727443447609, 'colsample_bytree': 0.9682410787270357, 'reg_lambda': 83.70940749007843, 'reg_alpha': 29.602201491643136, 'min_child_weight': 12.10403420144089}\n",
      "CV Fold 1 rmse:0.72653 - 125 rounds - it took 38 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72074\n",
      "Last eval:  [0.9802376329369777, 3, 0.9499727443447609, 0.9682410787270357, 83.70940749007843, 29.602201491643136, 12.10403420144089]  - Score  0.7265315515986072\n",
      "Current iter:  38  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.021633588861131273, 'max_depth': 3, 'subsample': 0.1500972169253243, 'colsample_bytree': 0.9707419046662611, 'reg_lambda': 82.3921484309917, 'reg_alpha': 91.28928810517053, 'min_child_weight': 26.836325622631755}\n",
      "CV Fold 1 rmse:0.72370 - 7252 rounds - it took 698 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72074\n",
      "Last eval:  [0.021633588861131273, 3, 0.1500972169253243, 0.9707419046662611, 82.3921484309917, 91.28928810517053, 26.836325622631755]  - Score  0.723702327049536\n",
      "Current iter:  39  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.0439988862465263, 'max_depth': 4, 'subsample': 0.9870107876334038, 'colsample_bytree': 0.7893458127339418, 'reg_lambda': 14.322036317655842, 'reg_alpha': 99.99630149255663, 'min_child_weight': 9.254541466647611}\n",
      "CV Fold 1 rmse:0.72084 - 2682 rounds - it took 637 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72074\n",
      "Last eval:  [0.0439988862465263, 4, 0.9870107876334038, 0.7893458127339418, 14.322036317655842, 99.99630149255663, 9.254541466647611]  - Score  0.7208421850845559\n",
      "Current iter:  40  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.5280502555794364, 'max_depth': 3, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 13.476260800157242, 'reg_alpha': 100.0, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.72047 - 582 rounds - it took 33 secs\n",
      "CV Fold 2 rmse:0.71784 - 517 rounds - it took 31 secs\n",
      "CV Fold 3 rmse:0.72246 - 828 rounds - it took 42 secs\n",
      "CV Fold 4 rmse:0.71611 - 464 rounds - it took 28 secs\n",
      "CV Fold 5 rmse:0.71706 - 588 rounds - it took 33 secs\n",
      "Last eval:  [0.5280502555794364, 3, 1.0, 0.1, 13.476260800157242, 100.0, 30.0]  - Score  0.7187871971716655\n",
      "Current iter:  41  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 1.0, 'max_depth': 1, 'subsample': 0.3919613634899708, 'colsample_bytree': 1.0, 'reg_lambda': 12.145793038056294, 'reg_alpha': 100.0, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.72242 - 1392 rounds - it took 86 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72059\n",
      "Last eval:  [1.0, 1, 0.3919613634899708, 1.0, 12.145793038056294, 100.0, 30.0]  - Score  0.7224247509306544\n",
      "Current iter:  42  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.2549930687672637, 'max_depth': 1, 'subsample': 0.12598258018943126, 'colsample_bytree': 0.9794405043791686, 'reg_lambda': 55.12502147481078, 'reg_alpha': 5.7231108384197, 'min_child_weight': 23.382177049127986}\n",
      "CV Fold 1 rmse:0.72540 - 1225 rounds - it took 51 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72059\n",
      "Last eval:  [0.2549930687672637, 1, 0.12598258018943126, 0.9794405043791686, 55.12502147481078, 5.7231108384197, 23.382177049127986]  - Score  0.7253995387787272\n",
      "Current iter:  43  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.030394282743219446, 'max_depth': 5, 'subsample': 0.10218741064613542, 'colsample_bytree': 0.14518144144322742, 'reg_lambda': 79.34038293118849, 'reg_alpha': 8.98010210468164, 'min_child_weight': 24.89498038917086}\n",
      "CV Fold 1 rmse:0.72118 - 2152 rounds - it took 132 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72059\n",
      "Last eval:  [0.030394282743219446, 5, 0.10218741064613542, 0.14518144144322742, 79.34038293118849, 8.98010210468164, 24.89498038917086]  - Score  0.7211783708031826\n",
      "Current iter:  44  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.42798824377283656, 'max_depth': 8, 'subsample': 0.9477853503738463, 'colsample_bytree': 0.13947677436205938, 'reg_lambda': 49.682465692553095, 'reg_alpha': 1.9205604479882625, 'min_child_weight': 12.71177167944459}\n",
      "CV Fold 1 rmse:0.72725 - 65 rounds - it took 19 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72059\n",
      "Last eval:  [0.42798824377283656, 8, 0.9477853503738463, 0.13947677436205938, 49.682465692553095, 1.9205604479882625, 12.71177167944459]  - Score  0.7272467637738025\n",
      "Current iter:  45  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.015715579296693705, 'max_depth': 5, 'subsample': 0.24719535756617528, 'colsample_bytree': 0.9949696446510016, 'reg_lambda': 93.83086734659288, 'reg_alpha': 13.247165501095662, 'min_child_weight': 29.670252800674287}\n",
      "CV Fold 1 rmse:0.72224 - 2870 rounds - it took 552 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72059\n",
      "Last eval:  [0.015715579296693705, 5, 0.24719535756617528, 0.9949696446510016, 93.83086734659288, 13.247165501095662, 29.670252800674287]  - Score  0.7222441583373496\n",
      "Current iter:  46  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9635706811547448, 'max_depth': 5, 'subsample': 0.943371220225146, 'colsample_bytree': 0.10075956472294707, 'reg_lambda': 78.25118981241367, 'reg_alpha': 84.34282382875739, 'min_child_weight': 28.60256840112854}\n",
      "CV Fold 1 rmse:0.72149 - 187 rounds - it took 22 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72059\n",
      "Last eval:  [0.9635706811547448, 5, 0.943371220225146, 0.10075956472294707, 78.25118981241367, 84.34282382875739, 28.60256840112854]  - Score  0.7214873157373795\n",
      "Current iter:  47  - Best Score  0.7177736524943502  - Best Args:  [0.04750495343906687, 4, 0.868163246780575, 0.11711450684675054, 80.20861835126621, 84.49765104431548, 12.063897523196292]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 6, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 20.733696423681973, 'reg_alpha': 57.3382961174106, 'min_child_weight': 30.0}\n",
      "CV Fold 1 rmse:0.71910 - 9972 rounds - it took 1040 secs\n",
      "CV Fold 2 rmse:0.71648 - 9972 rounds - it took 1044 secs\n",
      "CV Fold 3 rmse:0.72111 - 8847 rounds - it took 933 secs\n",
      "CV Fold 4 rmse:0.71469 - 9342 rounds - it took 993 secs\n",
      "CV Fold 5 rmse:0.71571 - 9443 rounds - it took 1002 secs\n",
      "Last eval:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]  - Score  0.7174185819307809\n",
      "Current iter:  48  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.029800018814920032, 'max_depth': 8, 'subsample': 0.1718531143365018, 'colsample_bytree': 0.13292990915566316, 'reg_lambda': 17.963502667456236, 'reg_alpha': 0.31795660264677406, 'min_child_weight': 26.337924935489532}\n",
      "CV Fold 1 rmse:0.72235 - 965 rounds - it took 92 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72040\n",
      "Last eval:  [0.029800018814920032, 8, 0.1718531143365018, 0.13292990915566316, 17.963502667456236, 0.31795660264677406, 26.337924935489532]  - Score  0.7223508538259944\n",
      "Current iter:  49  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.20911329754194904, 'max_depth': 5, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 7.085675725306103, 'reg_alpha': 58.21007781236761, 'min_child_weight': 1.0}\n",
      "CV Fold 1 rmse:0.71919 - 700 rounds - it took 68 secs\n",
      "CV Fold 2 rmse:0.71681 - 758 rounds - it took 73 secs\n",
      "CV Fold 3 rmse:0.72125 - 714 rounds - it took 70 secs\n",
      "CV Fold 4 rmse:0.71466 - 928 rounds - it took 85 secs\n",
      "CV Fold 5 rmse:0.71555 - 721 rounds - it took 70 secs\n",
      "Last eval:  [0.20911329754194904, 5, 1.0, 0.1, 7.085675725306103, 58.21007781236761, 1.0]  - Score  0.7174913918595729\n",
      "Current iter:  50  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.1155292095180817, 'max_depth': 1, 'subsample': 0.974860363163121, 'colsample_bytree': 0.9711612037507101, 'reg_lambda': 15.791943929830863, 'reg_alpha': 98.5115105766098, 'min_child_weight': 28.98188197071029}\n",
      "CV Fold 1 rmse:0.72460 - 9998 rounds - it took 802 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72018\n",
      "Last eval:  [0.1155292095180817, 1, 0.974860363163121, 0.9711612037507101, 15.791943929830863, 98.5115105766098, 28.98188197071029]  - Score  0.7245975564650092\n",
      "Current iter:  51  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.8507441131416335, 'max_depth': 1, 'subsample': 0.1272228397009001, 'colsample_bytree': 0.9645739697051062, 'reg_lambda': 67.32755737689268, 'reg_alpha': 3.2319957246445217, 'min_child_weight': 8.518374983284925}\n",
      "CV Fold 1 rmse:0.72810 - 423 rounds - it took 20 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72018\n",
      "Last eval:  [0.8507441131416335, 1, 0.1272228397009001, 0.9645739697051062, 67.32755737689268, 3.2319957246445217, 8.518374983284925]  - Score  0.7280962304667526\n",
      "Current iter:  52  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.2857964089466379, 'max_depth': 5, 'subsample': 1.0, 'colsample_bytree': 0.1, 'reg_lambda': 100.0, 'reg_alpha': 100.0, 'min_child_weight': 5.162041520199996}\n",
      "CV Fold 1 rmse:0.71994 - 654 rounds - it took 53 secs\n",
      "CV Fold 2 rmse:0.71751 - 509 rounds - it took 46 secs\n",
      "CV Fold 3 rmse:0.72209 - 661 rounds - it took 55 secs\n",
      "CV Fold 4 rmse:0.71578 - 526 rounds - it took 47 secs\n",
      "CV Fold 5 rmse:0.71670 - 639 rounds - it took 53 secs\n",
      "Last eval:  [0.2857964089466379, 5, 1.0, 0.1, 100.0, 100.0, 5.162041520199996]  - Score  0.7184041412011719\n",
      "Current iter:  53  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.9989869842830074, 'max_depth': 1, 'subsample': 0.947658956386034, 'colsample_bytree': 0.11524725949766454, 'reg_lambda': 31.846790668992558, 'reg_alpha': 91.256292057894, 'min_child_weight': 1.461887055002872}\n",
      "CV Fold 1 rmse:0.72344 - 2132 rounds - it took 70 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72004\n",
      "Last eval:  [0.9989869842830074, 1, 0.947658956386034, 0.11524725949766454, 31.846790668992558, 91.256292057894, 1.461887055002872]  - Score  0.7234367847908081\n",
      "Current iter:  54  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.02340445902262433, 'max_depth': 3, 'subsample': 0.2193525519009283, 'colsample_bytree': 0.11622363563229857, 'reg_lambda': 25.00108735166013, 'reg_alpha': 97.37104040674957, 'min_child_weight': 23.66312532586119}\n",
      "CV Fold 1 rmse:0.72352 - 9994 rounds - it took 421 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72004\n",
      "Last eval:  [0.02340445902262433, 3, 0.2193525519009283, 0.11622363563229857, 25.00108735166013, 97.37104040674957, 23.66312532586119]  - Score  0.7235171284323082\n",
      "Current iter:  55  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.01, 'max_depth': 5, 'subsample': 0.8349772213388623, 'colsample_bytree': 0.4215627543761439, 'reg_lambda': 77.56984228694216, 'reg_alpha': 38.63143923470717, 'min_child_weight': 1.0}\n",
      "CV Fold 1 rmse:0.71994 - 7772 rounds - it took 1443 secs\n",
      "CV Fold 2 rmse:0.71731 - 7103 rounds - it took 1339 secs\n",
      "CV Fold 3 rmse:0.72190 - 6160 rounds - it took 1137 secs\n",
      "CV Fold 4 rmse:0.71549 - 7795 rounds - it took 1441 secs\n",
      "CV Fold 5 rmse:0.71630 - 7896 rounds - it took 1460 secs\n",
      "Last eval:  [0.01, 5, 0.8349772213388623, 0.4215627543761439, 77.56984228694216, 38.63143923470717, 1.0]  - Score  0.7181880026383476\n",
      "Current iter:  56  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.900526642605176, 'max_depth': 8, 'subsample': 0.9775696803383567, 'colsample_bytree': 0.11506093269487798, 'reg_lambda': 20.280119818026836, 'reg_alpha': 83.31276971377733, 'min_child_weight': 24.149296404645217}\n",
      "CV Fold 1 rmse:0.72216 - 120 rounds - it took 21 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72000\n",
      "Last eval:  [0.900526642605176, 8, 0.9775696803383567, 0.11506093269487798, 20.280119818026836, 83.31276971377733, 24.149296404645217]  - Score  0.7221609011991172\n",
      "Current iter:  57  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.0831088244053642, 'max_depth': 8, 'subsample': 0.21576004027259577, 'colsample_bytree': 0.996020503998949, 'reg_lambda': 14.974562898980661, 'reg_alpha': 11.67041848160041, 'min_child_weight': 3.7319817263461816}\n",
      "CV Fold 1 rmse:0.72779 - 194 rounds - it took 74 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72000\n",
      "Last eval:  [0.0831088244053642, 8, 0.21576004027259577, 0.996020503998949, 14.974562898980661, 11.67041848160041, 3.7319817263461816]  - Score  0.7277866137177345\n",
      "Current iter:  58  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.992494997074066, 'max_depth': 1, 'subsample': 0.7682815600506987, 'colsample_bytree': 0.21584019719329792, 'reg_lambda': 5.272289066660176, 'reg_alpha': 8.867893080079327, 'min_child_weight': 26.216616975383495}\n",
      "CV Fold 1 rmse:0.72254 - 1076 rounds - it took 40 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72000\n",
      "Last eval:  [0.992494997074066, 1, 0.7682815600506987, 0.21584019719329792, 5.272289066660176, 8.867893080079327, 26.216616975383495]  - Score  0.7225429748998676\n",
      "Current iter:  59  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n",
      "\n",
      "Testing:  {'learning_rate': 0.5607941183604543, 'max_depth': 3, 'subsample': 0.9362670117420269, 'colsample_bytree': 0.9914733862090448, 'reg_lambda': 57.76940218442397, 'reg_alpha': 94.13090274935499, 'min_child_weight': 7.738145774333029}\n",
      "CV Fold 1 rmse:0.72210 - 390 rounds - it took 96 secs\n",
      "Early stopping for under-performing fold: threshold is 0.72000\n",
      "Last eval:  [0.5607941183604543, 3, 0.9362670117420269, 0.9914733862090448, 57.76940218442397, 94.13090274935499, 7.738145774333029]  - Score  0.7220974142683002\n",
      "Current iter:  60  - Best Score  0.7174185819307809  - Best Args:  [0.01, 6, 1.0, 0.1, 20.733696423681973, 57.3382961174106, 30.0]\n"
     ]
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "\n",
    "gp_round = gp_minimize(func=objective,\n",
    "                       x0=x0,              # already examined values for x\n",
    "                       y0=y0,              # observed values for x0\n",
    "                       dimensions=space,\n",
    "                       acq_func='gp_hedge',\n",
    "                       n_calls=30,\n",
    "                       n_initial_points=0,\n",
    "                       callback=[onstep],\n",
    "                       random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "88327011",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T04:44:46.042258Z",
     "iopub.status.busy": "2021-09-09T04:44:46.038271Z",
     "iopub.status.idle": "2021-09-09T04:44:46.046868Z",
     "shell.execute_reply": "2021-09-09T04:44:46.046368Z"
    },
    "papermill": {
     "duration": 0.393311,
     "end_time": "2021-09-09T04:44:46.047015",
     "exception": false,
     "start_time": "2021-09-09T04:44:45.653704",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60\n"
     ]
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "print(len(x0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9ec0248f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-09T04:44:46.817333Z",
     "iopub.status.busy": "2021-09-09T04:44:46.816673Z",
     "iopub.status.idle": "2021-09-09T04:44:46.820870Z",
     "shell.execute_reply": "2021-09-09T04:44:46.820216Z"
    },
    "papermill": {
     "duration": 0.39101,
     "end_time": "2021-09-09T04:44:46.821020",
     "exception": false,
     "start_time": "2021-09-09T04:44:46.430010",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.71742\n",
      "Best hyperparameters:\n",
      "learning_rate             : 0.01\n",
      "max_depth                 : 6\n",
      "subsample                 : 1.0\n",
      "colsample_bytree          : 0.1\n",
      "reg_lambda                : 20.733696423681973\n",
      "reg_alpha                 : 57.3382961174106\n",
      "min_child_weight          : 30.0\n"
     ]
    }
   ],
   "source": [
    "print(f\"Best score: {gp_round.fun:0.5f}\")\n",
    "print(\"Best hyperparameters:\")\n",
    "for sp, x in zip(gp_round.space, gp_round.x):\n",
    "    print(f\"{sp.name:25} : {x}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 29639.185306,
   "end_time": "2021-09-09T04:44:49.169863",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-09-08T20:30:49.984557",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
